#!/usr/bin/env ruby
#encoding: utf-8

module Crawler
  
  require 'observer'
  require 'backend/crawler/html_parser'
  
  class WebCrawler
    
    include Observable
    
    def crawl
      thread = Thread.new do
        while true
          retryCnt = 0
          begin
            items = HtmlParser::crawl
            unless items.nil?
              changed
              notify_observers items
            end
            
            sleep(15 * 60)
          rescue => e
            puts "Error: #{e.class}, #{e.message}"
            retryCnt += retryCnt
            retry if retryCnt < 4
          end
        end
      end
      thread
    end
    
  end
  
  if __FILE__ == $0
    puts 'hello'
    crawler = WebCrawler.new
    
    require 'backend/data/data_processor'
    DB::DataProcessor.new crawler
    
    thread = crawler.crawl
    puts 'create a new thread'
    
    threads = []
    threads << thread
    
    require 'thwait'
    ThreadsWait.all_waits(*threads)
  end
  
end
